
# 翻页

'''
    需求：使用爬虫抓取全栈数据，数据总页数是后端返回的，怎么自动翻页
    page = 1
    total_page = 2
    while page <= total_page:
        print(page)
        print(f"第{page}页数据采集完毕")
        total_page = 10
        page += 1
'''


import requests

page = 1
total_page = 2
def get_data():
    global page, total_page
    while page <= total_page:
        # 地址
        url = "https://campusgateway.51job.com/jobone-api/school/getSchoolByCity"

        # 查询参数
        params = {
            "pageIndex": page,
            "pageSize": "21",
            #"city": "北京"
        }

        # 请求头
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36"
        }

        # 请求发送，响应接收
        response = requests.get(url, params=params, headers=headers)

        # 提取响应
        # print(response.text)

        # json格式数据，可以转字典，进行key value提取
        items = response.json()
        for data in items['data']['records']:
            # print(data)
            name = data.get('schname')
            print(name)

        print(f'目前采集第{page}页的数据')
        total_page = items.get('data')['pages']
        page += 1

get_data()